import pandas as pd
import numpy as np
from scipy import sparse as ssp
from sklearn.preprocessing import StandardScaler,MinMaxScaler,RobustScaler
from sklearn.datasets import dump_svmlight_file,load_svmlight_file
import warnings
from sklearn.linear_model import LogisticRegression,RidgeClassifier,Ridge
from sklearn.metrics import accuracy_score

seed = 1024
np.random.seed(seed)


path = '../data/'

train_fea = pd.read_pickle(path+'train_X.pkl')
val_fea = pd.read_pickle(path+'valid_X.pkl')
dev_fea = pd.read_pickle(path+'dev_X.pkl')

train_tfidf= pd.read_pickle(path+'train_context_tfidf.pkl')
val_tfidf = pd.read_pickle(path+'valid_context_tfidf.pkl')
dev_tfidf = pd.read_pickle(path+'dev_context_tfidf.pkl')


X_train=ssp.hstack([train_fea,train_tfidf]).tocsr()

import gc
gc.collect()

X_valid=ssp.hstack([val_fea,val_tfidf]).tocsr()

X_dev = ssp.hstack([dev_fea,dev_tfidf]).tocsr()

y_label =pd.read_pickle(path+'train.pkl')['label'].values
y_valid = pd.read_pickle(path+'valid.pkl')['label'].values
label_test=np.zeros(X_dev.shape[0])


dump_svmlight_file(X_train,y_label,path+"X_train_v1.svm")
dump_svmlight_file(X_dev,label_test,path+"X_valid_v1.svm")


#
X_train,y_train=load_svmlight_file(path+'X_train_v1.svm')

lr=LogisticRegression(n_jobs=7,random_state=1123,C=1.0)
lr.fit(X_train,y_train)


X_valid,_ = load_svmlight_file(path+'X_valid_v1.svm')

y_pred = lr.predict(X_valid)


y_t = (y_pred+0.5).astype(int)
acc =  accuracy_score(y_valid,y_t)

print('xgb model the accuracy on the dev set is : {}%'.format(round(acc* 100,2)))







